#============================================================
# Create Time:			2020-06-15 09:06:35
# Last modify:			2020-06-28 17:47:18
# Writer:				Wenhao	1795902848@qq.com
# File Name:			final.py
# File Type:			PY Source File
# Tool:					Mac -- vim & python
# Information:			
#============================================================
from scipy.io import loadmat
import numpy as np
import os, pickle
import sys
sys.path.append('/Users/apple/Documents/Coding/Python/Mail')
from SendMail import SendMail as MailSender

class FD_Kin(object):
    '''father daughter relationship judger
        本质上它是个KNN 分类器。
        要算两张图片的距离d，就要知道矩阵W
        W是max(J)优化的结果，max(J)有着闭式解
        算H的特征向量，取前k个最大的特征值对应的特征向量，这样就有了矩阵W
        于是就可以计算两个图片的距离了。接下来就可以用KNN来进行分类。
    '''
    def __init__(this, k, l):
        ''' 
        mat_data.keys() = [
                '__header__', '__version__', '__globals__',
               'ux', 'fold', 'matches', 'idxa', 'idxb']
        ux是数据，共有500张图片的lbp特征。
        idxa和idxb是500个图片对。
        matches是一个一维向量，由0或1表示每个图片对是否是亲子关系。
        fold用来区分5个折的数据。
       '''
        super(FD_Kin, this).__init__()
        this.mat_data = loadmat('./fd_LBP.mat')     #->dict
        this.k = k      #k nearst neighbor
        this.l = l      #l
        this.W_name = './pickle/k_%d_W.pickle'%k    #Module name
        this.dist_name = './pickle/%d_dist.pickle'%(k)
        return None

    def _eig_vector(this, H):
        '''返回的还是行向量'''
        eig = np.linalg.eig(H)
        eig_num = eig[0]
        eig_vec = eig[1].T
        eig_zipped = [[a, b] for a, b in zip(eig_num, eig_vec)]
        eig_zipped.sort(key=lambda x:x[0], reverse=True)
        eig_vec_sorted = [b for a, b in eig_zipped]
        eig_vec = np.array(eig_vec_sorted).T
        return eig_vec

    def get_W(this):
        '''
        Shape of initial W is (dim, l)
        '''
        matched_idx = [ i
                for i in range(len(this.mat_data.get('matches')))
                if this.mat_data.get('matches')[i] == 1
            ]
        dim = len(this.mat_data.get('ux')[0])
        N = len(matched_idx)    #match的图像对数
        W = np.ones(shape=[dim, this.l])    #初始的W，用幺矩阵初始
        l2_W = np.linalg.norm(W)    #W的l2范数
        while True:
            H1 = np.zeros(shape=[dim, dim])
            for i in matched_idx:
                print('H1 dealing', i)
                distances = []
                for j in matched_idx:
                    if i == j:          #为了确保两个图片不是亲子关系
                        continue
                    idx_x = this.mat_data.get('idxa')[i] - 1
                    idx_y = this.mat_data.get('idxb')[j] - 1
                    vec_x = np.array(this.mat_data.get('ux')[idx_x])
                    vec_y = np.array(this.mat_data.get('ux')[idx_y])

                    #按照W来计算图像特征之间的距离
                    vec_subbed = vec_x - vec_y      #shape = (1, dim)

                    #(1, dim) * (dim, l)
                    vec_subbed = np.matmul(vec_subbed, W)    #shape = (1, l)
                    distance = np.linalg.norm(vec_subbed)      #L2范数
                    distances.append([idx_x, idx_y, distance])
                distances.sort(key=lambda x:x[2], reverse=False)    #根据y和x的距离排序
                for idx_x, idx_y, _ in distances[:this.k]:      #取出前k个向量
                    vec_x = np.array(this.mat_data.get('ux')[idx_x])
                    vec_y = np.array(this.mat_data.get('ux')[idx_y])
                    vec_subbed = np.array(vec_x - vec_y)    #默认是行向量
                    h1 = np.matmul(vec_subbed.T, vec_subbed)
                    H1 += h1
            H1 /= N*this.k


            H2 = np.zeros(shape=[dim, dim])
            for i in matched_idx:
                print('H2 dealing', i)
                distances = []
                for j in matched_idx:
                    if i == j:          #为了确保两个图片不是亲子关系
                        continue
                    idx_x = this.mat_data.get('idxb')[i] - 1
                    idx_y = this.mat_data.get('idxa')[j] - 1
                    vec_x = np.array(this.mat_data.get('ux')[idx_x])
                    vec_y = np.array(this.mat_data.get('ux')[idx_y])

                    #按照W来计算图像特征之间的距离
                    vec_subbed = vec_x - vec_y      #shape = (1, dim)

                    #(1, dim) * (dim, l)
                    vec_subbed = np.matmul(vec_subbed, W)    #shape = (1, l)

                    distance = np.linalg.norm(vec_subbed)      #L2范数
                    distances.append([idx_x, idx_y, distance])
                distances.sort(key=lambda x:x[2], reverse=False)    #根据y和x的距离排序
                for idx_x, idx_y, _ in distances[:this.k]:      #取出前k个向量
                    vec_x = np.array(this.mat_data.get('ux')[idx_x])
                    vec_y = np.array(this.mat_data.get('ux')[idx_y])
                    vec_subbed = np.array(vec_x - vec_y)    #默认是行向量
                    h2 = np.matmul(vec_subbed.T, vec_subbed)
                    H2 += h2
            H2 /= N*this.k


            #遍历所有匹配的图片对
            H3 = np.zeros(shape=[dim, dim])
            for i in matched_idx:
                print('H3 dealing', i)
                idx_x = this.mat_data.get('idxa')[i] - 1
                idx_y = this.mat_data.get('idxb')[i] - 1
                vec_x = np.array(this.mat_data.get('ux')[idx_x])
                vec_y = np.array(this.mat_data.get('ux')[idx_y])
                vec_subbed = vec_x - vec_y
                h3 = np.matmul(vec_subbed.T, vec_subbed)
                H3 += h3
            H3 /= N


            H = H1 + H2 - H3    #shape = (dim, dim)
            eig_vecs = this._eig_vector(H)
            eig_vecs = [vec[:this.l] for vec in eig_vecs]   #shape = (dim, l)
            W_new = np.array(eig_vecs)
            l2_W_new = np.linalg.norm(W_new)

            delta = abs(l2_W - l2_W_new)
            print('delta of W is', delta)
            if(delta < 1e-3):
                return W_new
            else:
                W = W_new
                l2_W = l2_W_new

        return None

    def get_avg_distance(this, W):
        '''要在W已经计算出来之后才能计算平均的匹配图片对的距离。
        '''
        matched_idx = [ i
                for i in range(len(this.mat_data.get('matches')))
                if this.mat_data.get('matches')[i] == 1
            ]
        distances = []
        for i in matched_idx:
            idx_x = this.mat_data.get('idxa')[i]-1
            idx_y = this.mat_data.get('idxb')[i]-1
            vec_x = np.array(this.mat_data.get('ux')[idx_x])
            vec_y = np.array(this.mat_data.get('ux')[idx_y])
            vec_subbed = vec_x - vec_y      #shape = (1, dim)

            #按照W来计算图像特征之间的距离
            #(1, dim) * (dim, l)
            vec_subbed = np.matmul(vec_subbed, W)    #shape = (1, l)
            distance = np.linalg.norm(vec_subbed)      #L2范数
            distances.append(distance)
        return np.average(distances)

    def check(this, fold:int, W, distance):
        fold_idx = [ i
            for i in range(len(this.mat_data.get('fold')))
            if this.mat_data.get('fold')[i] == fold
        ]
        tp, fp = 0, 0
        for i in fold_idx:
            print('Fold: %d, Dealing: %d'%(fold, i))
            idx_x = this.mat_data.get('idxa')[i] - 1
            idx_y = this.mat_data.get('idxb')[i] - 1
            vec_x = this.mat_data.get('ux')[idx_x]
            vec_y = this.mat_data.get('ux')[idx_y]
            vec_subbed = vec_x - vec_y
            vec_subbed = np.matmul(vec_subbed, W)
            tmp_dist = np.linalg.norm(vec_subbed)      #L2范数
            if(tmp_dist <= distance):
                if(idx_x%2==0 and idx_y==idx_x+1):
                    tp += 1
                elif(idx_x%2==1 and idx_y==idx_x-1):
                    tp += 1
                else:
                    fp += 1
        tp, fp = tp/50, fp/50
        return [tp, fp]

    def run(this):
        if(not os.path.exists(this.W_name)):
            print('There is no W yet, training W for you')
            W = this.get_W()
            print('Train W done')
            print('Saving W to %s'%(this.W_name))
            with open(this.W_name, 'wb') as f:
                pickle.dump(W, f)
            print('Saved W to %s'%(this.W_name))
        else:
            with open(this.W_name, 'rb') as f:
                W = pickle.load(f)
            print('Loaded W from %s successfully'%(this.W_name))
        print('Now checking')

        if(not os.path.exists(this.dist_name)):
            print('average distance not exist, calculating for you')
            distance = this.get_avg_distance(W)
            with open(this.dist_name, 'wb') as f:
                pickle.dump(distance, f)
        else:
            print('loaded average distance from', this.dist_name)
            with open(this.dist_name, 'rb') as f:
                distance = pickle.load(f)

        tp, fp = 0, 0
        for fold in range(5):
            print('Processing fold', fold+1)
            _tp, _fp = this.check(fold+1, W, distance)
            tp += _tp
            fp += _fp
        tp /= 5
        fp /= 5
        output = 'k: %d\nTrue postive: %.4f\nFalse postive: %.4f\n\n'%(this.k, tp, fp)
        with open('output.txt', 'a') as f:
            f.write(output)
        print('True postive', tp)
        print('False postive', fp)

        return output

if __name__ == '__main__':
    def final():
        for k in range(1, 16):
            fd_kin = FD_Kin(k, 30)
            output = fd_kin.run()
            sender = MailSender()
            sender.send(Subject='Kinship check', txt=output)
            sender.end()
    def test():
        k = 4
        fd_kin = FD_Kin(k, 30)
        output = fd_kin.run()
        sender = MailSender()
        sender.send(Subject='Kinship check', txt=output)
        sender.end()

    final()
    #test()
